<!DOCTYPE html>
<html>
<head><meta name="generator" content="Hexo 3.9.0">
  <meta charset="utf-8">
  
  <title>Spark-Shuffle原理 | 月迹的博客</title>
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta name="description" content="前言Spark Shuffle是大众讨论的比较多的话题了。它是Spark任务执行过程中最为重要的过程之一。那么什么是Shuffle呢？Shuffle一般被翻译成数据混洗，是类MapReduce分布式计算框架独有的机制，也是这类分布式计算框架最重要的执行机制。接下来会按照两个层面来谈谈Shuffle机制。分别为：  逻辑层面 物理层面  逻辑层面主要是从RDD的血缘出发，从DAG的角度来讲解Shuf">
<meta name="keywords" content="shuffle,spark">
<meta property="og:type" content="article">
<meta property="og:title" content="Spark-Shuffle原理">
<meta property="og:url" content="https://xyueji.github.io/Spark-Shuffle原理.html">
<meta property="og:site_name" content="月迹的博客">
<meta property="og:description" content="前言Spark Shuffle是大众讨论的比较多的话题了。它是Spark任务执行过程中最为重要的过程之一。那么什么是Shuffle呢？Shuffle一般被翻译成数据混洗，是类MapReduce分布式计算框架独有的机制，也是这类分布式计算框架最重要的执行机制。接下来会按照两个层面来谈谈Shuffle机制。分别为：  逻辑层面 物理层面  逻辑层面主要是从RDD的血缘出发，从DAG的角度来讲解Shuf">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/FFA9D34B-8365-41BC-8075-3AEE04D9F8F9.png">
<meta property="og:image" content="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/5B9FADD6-00EE-4441-8092-E5CC2FF64A27.png">
<meta property="og:image" content="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/FCF2BEB0-6A18-4EA0-9F22-46B0E3AA0952.png">
<meta property="og:image" content="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/2F861E67-2AD8-422C-B3C0-5AF1885FBB60.png">
<meta property="og:image" content="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/3D2885AA-8879-41AF-8E7B-D5BA7CE09B9F.png">
<meta property="og:updated_time" content="2020-10-14T11:49:19.097Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Spark-Shuffle原理">
<meta name="twitter:description" content="前言Spark Shuffle是大众讨论的比较多的话题了。它是Spark任务执行过程中最为重要的过程之一。那么什么是Shuffle呢？Shuffle一般被翻译成数据混洗，是类MapReduce分布式计算框架独有的机制，也是这类分布式计算框架最重要的执行机制。接下来会按照两个层面来谈谈Shuffle机制。分别为：  逻辑层面 物理层面  逻辑层面主要是从RDD的血缘出发，从DAG的角度来讲解Shuf">
<meta name="twitter:image" content="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/FFA9D34B-8365-41BC-8075-3AEE04D9F8F9.png">
  
    <link rel="alternative" href="/atom.xml" title="月迹的博客" type="application/atom+xml">
  
  
    <link rel="icon" href="/img/favicon.png">
  
  
      <link rel="stylesheet" href="//cdn.bootcss.com/animate.css/3.5.0/animate.min.css">
  
  <link rel="stylesheet" href="css/style.css">
  <link rel="stylesheet" href="/font-awesome/css/font-awesome.min.css">
  <link rel="apple-touch-icon" href="/apple-touch-icon.png">
  
  
      <link rel="stylesheet" href="/fancybox/jquery.fancybox.css">
  
    
  <!-- 加载特效 -->
    <script src="/js/pace.js"></script>
    <link href="/css/pace/pace-theme-flash.css" rel="stylesheet">
  <script>
      var yiliaConfig = {
          rootUrl: '/',
          fancybox: true,
          animate: true,
          isHome: false,
          isPost: true,
          isArchive: false,
          isTag: false,
          isCategory: false,
          open_in_new: false
      }
  </script>
    <script src="https://cdn.bootcdn.net/ajax/libs/gitalk/1.6.2/gitalk.min.js"></script>
    <script>
        (function(){
            var bp = document.createElement('script');
            var curProtocol = window.location.protocol.split(':')[0];
            if (curProtocol === 'https') {
                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
            }
            else {
                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
            }
            var s = document.getElementsByTagName("script")[0];
            s.parentNode.insertBefore(bp, s);
        })();
    </script>
</head>
</html>
<body>
  <div id="container">
    <div class="left-col">
    <div class="overlay"></div>
<div class="intrude-less">
    <header id="header" class="inner">
        <a href="/" class="profilepic">
            
            <img lazy-src="/img/head.png" class="js-avatar">
            
        </a>

        <hgroup>
          <h1 class="header-author"><a href="/" title="Hi Mate">月 迹</a></h1>
        </hgroup>

        
        <p class="header-subtitle">用代码书写人生</p>
        
        
            <input type="text" class="search" id="search" style="text-align: center" placeholder="搜点什么...">
        
        
            <div id="switch-btn" class="switch-btn">
                <div class="icon">
                    <div class="icon-ctn">
                        <div class="icon-wrap icon-house" data-idx="0">
                            <div class="birdhouse"></div>
                            <div class="birdhouse_holes"></div>
                        </div>
                        <div class="icon-wrap icon-ribbon hide" data-idx="1">
                            <div class="ribbon"></div>
                        </div>
                        
                        <div class="icon-wrap icon-link hide" data-idx="2">
                            <div class="loopback_l"></div>
                            <div class="loopback_r"></div>
                        </div>
                        
                        
                        <div class="icon-wrap icon-me hide" data-idx="3">
                            <div class="user"></div>
                            <div class="shoulder"></div>
                        </div>
                        
                    </div>

                </div>
                <div class="tips-box hide">
                    <div class="tips-arrow"></div>
                    <ul class="tips-inner">
                        <li>菜单</li>
                        <li>标签</li>
                        
                        <li>友情链接</li>
                        
                        
                        <li>关于我</li>
                        
                    </ul>
                </div>
            </div>
        

        <div id="switch-area" class="switch-area">
            <div class="switch-wrap">
                <section class="switch-part switch-part1">
                    <nav class="header-menu">
                        <ul>
                        
                            <li><a href="index.html">博客首页</a></li>
                        
                            <li><a href="/archives">文章归档</a></li>
                        
                            <li><a href="/tags">文章统计</a></li>
                        
                            <li><a href="/message">交友留言</a></li>
                        
                            <li><a href="/navigation">资源导航</a></li>
                        
                            <li><a href="/about">关于作者</a></li>
                        
                        </ul>
                    </nav>
                    <nav class="header-nav">
                        <ul class="social">
                            
                                <a class="fl mail" target="_blank" href="http://mail.qq.com/cgi-bin/qm_share?t=qm_mailme&email=U2FhYmdrZGVjamITIiJ9MDw_" title="mail">mail</a>
                            
                                <a class="fl github" target="_blank" href="https://github.com/xyueji" title="github">github</a>
                            
                                <a class="fl rss" target="_blank" href="/atom.xml" title="rss">rss</a>
                            
                        </ul>
                    </nav>
                </section>

                
                <section class="switch-part switch-part2">
                    <div class="widget tagcloud" id="js-tagcloud">
                        <a href="tags/JanusGraph/" style="font-size: 10px;">JanusGraph</a> <a href="tags/alluxio/" style="font-size: 10px;">alluxio</a> <a href="tags/container/" style="font-size: 10px;">container</a> <a href="tags/datasource/" style="font-size: 10px;">datasource</a> <a href="tags/docker/" style="font-size: 15px;">docker</a> <a href="tags/docker-compose/" style="font-size: 15px;">docker-compose</a> <a href="tags/flink/" style="font-size: 15px;">flink</a> <a href="tags/free/" style="font-size: 10px;">free</a> <a href="tags/git/" style="font-size: 10px;">git</a> <a href="tags/hbase/" style="font-size: 10px;">hbase</a> <a href="tags/hbck/" style="font-size: 10px;">hbck</a> <a href="tags/idea/" style="font-size: 15px;">idea</a> <a href="tags/impala/" style="font-size: 10px;">impala</a> <a href="tags/iptables/" style="font-size: 10px;">iptables</a> <a href="tags/mac/" style="font-size: 10px;">mac</a> <a href="tags/mapreduce/" style="font-size: 10px;">mapreduce</a> <a href="tags/phoenix/" style="font-size: 15px;">phoenix</a> <a href="tags/presto/" style="font-size: 10px;">presto</a> <a href="tags/prometheus/" style="font-size: 10px;">prometheus</a> <a href="tags/pycharm/" style="font-size: 10px;">pycharm</a> <a href="tags/salt-api/" style="font-size: 10px;">salt-api</a> <a href="tags/shuffle/" style="font-size: 15px;">shuffle</a> <a href="tags/spark/" style="font-size: 15px;">spark</a> <a href="tags/springboot/" style="font-size: 10px;">springboot</a> <a href="tags/top/" style="font-size: 10px;">top</a> <a href="tags/vue/" style="font-size: 10px;">vue</a> <a href="tags/wordcount/" style="font-size: 10px;">wordcount</a> <a href="tags/修复工具/" style="font-size: 10px;">修复工具</a> <a href="tags/分片/" style="font-size: 10px;">分片</a> <a href="tags/基础知识/" style="font-size: 10px;">基础知识</a> <a href="tags/大数据/" style="font-size: 20px;">大数据</a> <a href="tags/大文件/" style="font-size: 10px;">大文件</a> <a href="tags/安装/" style="font-size: 10px;">安装</a> <a href="tags/开发规范/" style="font-size: 10px;">开发规范</a> <a href="tags/数据倾斜/" style="font-size: 10px;">数据倾斜</a> <a href="tags/测试/" style="font-size: 10px;">测试</a> <a href="tags/热门商品/" style="font-size: 10px;">热门商品</a> <a href="tags/端口/" style="font-size: 10px;">端口</a> <a href="tags/端口映射/" style="font-size: 10px;">端口映射</a> <a href="tags/系统表/" style="font-size: 10px;">系统表</a> <a href="tags/表重建/" style="font-size: 10px;">表重建</a>
                    </div>
                </section>
                

                
                <section class="switch-part switch-part3">
                    <div id="js-friends">
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="https://dayesahh.github.io/">DaYesahh</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.54tianzhisheng.cn/">zhisjeng</a>
                    
                    </div>
                </section>
                

                
                
                <section class="switch-part switch-part4">
                
                    <div id="js-aboutme">爱动漫、追小说、喜影视、迷于严谨和逻辑的编码之中！！！</div>
                </section>
                
            </div>
        </div>
    </header>
</div>

    </div>
    <div class="mid-col">
      <nav id="mobile-nav">
      <div class="overlay">
          <div class="slider-trigger"></div>
          <h1 class="header-author js-mobile-header hide"><a href="/" title="Me">月 迹</a></h1>
      </div>
    <div class="intrude-less">
        <header id="header" class="inner">
            <a href="/" class="profilepic">
                
                    <img lazy-src="/img/head.png" class="js-avatar">
                
            </a>
            <hgroup>
              <h1 class="header-author"><a href="/" title="Me">月 迹</a></h1>
            </hgroup>
            
            <p class="header-subtitle">用代码书写人生</p>
            
            <nav class="header-menu">
                <ul>
                
                    <li><a href="index.html">博客首页</a></li>
                
                    <li><a href="/archives">文章归档</a></li>
                
                    <li><a href="/tags">文章统计</a></li>
                
                    <li><a href="/message">交友留言</a></li>
                
                    <li><a href="/navigation">资源导航</a></li>
                
                    <li><a href="/about">关于作者</a></li>
                
                <div class="clearfix"></div>
                </ul>
            </nav>
            <nav class="header-nav">
                <div class="social">
                    
                        <a class="mail" target="_blank" href="http://mail.qq.com/cgi-bin/qm_share?t=qm_mailme&email=U2FhYmdrZGVjamITIiJ9MDw_" title="mail">mail</a>
                    
                        <a class="github" target="_blank" href="https://github.com/xyueji" title="github">github</a>
                    
                        <a class="rss" target="_blank" href="/atom.xml" title="rss">rss</a>
                    
                </div>
            </nav>
        </header>                
    </div>
</nav>
      <div class="body-wrap"><article id="post-Spark-Shuffle原理" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="" class="article-date">
      <time datetime="2020-10-14T11:49:19.000Z" itemprop="datePublished">2020-10-14</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 class="article-title" itemprop="name">
      Spark-Shuffle原理
    </h1>
  

      </header>
      
      <div class="article-info article-info-post">
        
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/spark/">spark</a>
    </div>


        
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/shuffle/">shuffle</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/spark/">spark</a></li></ul>
    </div>

        <div class="clearfix"></div>
      </div>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
        <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2><p>Spark Shuffle是大众讨论的比较多的话题了。它是Spark任务执行过程中最为重要的过程之一。那么什么是Shuffle呢？<br>Shuffle一般被翻译成数据混洗，是类MapReduce分布式计算框架独有的机制，也是这类分布式计算框架最重要的执行机制。接下来会按照两个层面来谈谈Shuffle机制。分别为：</p>
<ul>
<li>逻辑层面</li>
<li>物理层面</li>
</ul>
<p>逻辑层面主要是从RDD的血缘出发，从DAG的角度来讲解Shuffle，另外也会说明Spark容错机制。物理层面是从执行角度来剖析Shuffle是如何发生的</p>
<h3 id="1-RDD血缘与Spark容错"><a href="#1-RDD血缘与Spark容错" class="headerlink" title="1. RDD血缘与Spark容错"></a>1. RDD血缘与Spark容错</h3><a id="more"></a>
<p>从血缘角度出发就需先了解DAG，DAG被称之为有向无环图。在DAG中，最初的RDD被成为基础RDD，在基础RDD之上使用算子的过程中后续生成RDD被成为一个个子RDD，它们之间存在依赖关系。无论哪个RDD出现问题，都可以由这种依赖关系重新计算而成。这种依赖关系就被成为RDD血缘。血缘的表现方式主要分为宽依赖与窄依赖。</p>
<h4 id="1-1-窄依赖与宽依赖"><a href="#1-1-窄依赖与宽依赖" class="headerlink" title="1.1 窄依赖与宽依赖"></a>1.1 窄依赖与宽依赖</h4><blockquote>
<p>窄依赖的标准定义是：子RDD中的分区与父RDD中的分区只存在一对一的映射关系。<br>宽依赖的标准定义是：子RDD中分区与父RDD中分区存在一对多的映射关系。</p>
</blockquote>
<p><img src="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/FFA9D34B-8365-41BC-8075-3AEE04D9F8F9.png" alt="23abbd9ac0c0c032330c6ca695f98b37"></p>
<blockquote>
<p>从实际算子来说，map，filter，union等就是窄依赖，而groupByKey，reduceByKey就是典型的宽依赖。</p>
</blockquote>
<p>宽依赖还有个名字，叫shuffle依赖，也就是说宽依赖必然会发生在shuffle操作，shuffle也是划分stage的重要依据。而窄依赖由于不需要发生shuffle，所有计算都是在分区所在节点完成，类似于MR中的ChainMapper。所以说，在如果在程序中选取的算子形成了宽依赖，那么就必然会触发shuffle。<br><strong><em>所以当RDD在shuffle过程中某个分区出现了故障，只需要找到当前对应的Stage，而这个Stage必然是某个shuffle算子所进行划分的，找到了这个算子，就离定位错误原因越来越近了。</em></strong><br><img src="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/5B9FADD6-00EE-4441-8092-E5CC2FF64A27.png" alt="2613599ec6366c549584bd777e88523c"></p>
<blockquote>
<p>如上图所示，如果P1_0分区发生故障，那么按照依赖关系，则需要P0_0与P0_1的分区重算，P0_0与P0_1没有持久化，就会不断回溯，直到找到存在的父分区为止。至于为什么要持久化，原因就是当计算逻辑复杂时，就会引发依赖链过长，如果其中的某个RDD发生了问题。若没有进行持久化，Spark则会根据RDD血缘关系进行重头开始计算。重算显然对我们是代价极高的，所以用户可以在计算过程中，适当的调用RDD的checkpoint方法，保存好当前算好的中间结果，这样依赖关系链就会大大的缩短。因为checkpoint其实是会切断血缘的。这就是RDD的血缘机制即RDD的容错机制。</p>
</blockquote>
<p><strong><em>而Spark的容错机制则是主要分为资源管理平台的容错和Spark应用的容错。</em></strong></p>
<h4 id="1-2-Spark的容错机制"><a href="#1-2-Spark的容错机制" class="headerlink" title="1.2 Spark的容错机制"></a>1.2 Spark的容错机制</h4><p>Spark的应用是基于资源管理平台运行的，所以资源管理平台的容错也是Spark容错的一部分，如Yarn的ResourceManager HA机制。在Spark应用执行的过程中，可能会遇到以下几种失败情况：</p>
<ul>
<li>Driver出错</li>
<li>Executor出错</li>
<li>Task出错</li>
</ul>
<blockquote>
<p>Dirver执行失败是Spark应用最严重的一种情况，因为它标记着整个作业的执行失败，需要开发人员手动重启Driver。而Executor报错通常是所在的Worker出错，这时Driver就会将执行失败的Task调度到另一个Executor继续执行，重新执行的Task会根据RDD的依赖关系继续计算，并将报错的Executor从可用的Executor列表中移除。Spark会对执行失败的Task进行重试，重试3次后若依然出错，则整个作业就会失败。而在这个过程中，数据恢复和重试都依赖于RDD血缘机制。</p>
</blockquote>
<h3 id="2-Spark-Shuffle"><a href="#2-Spark-Shuffle" class="headerlink" title="2. Spark Shuffle"></a>2. Spark Shuffle</h3><p>很多算子都会引起RDD中的数据进行重分区，新的分区被创建，旧的分区被合并或者打碎，<strong><em>在重分区过程中，如果数据发生了跨节点移动，就被称为Shuffle。</em></strong> Spark对Shuffle的实现方式有两种：Hash Shuffle与Sort-based Shuffle，这其实是一个优化的过程。在较老的版本中，Spark Shuffle的方式可以通过spark.shuffle.manager配置项进行配置，而在最新的版本中，已经移除了该配置项，统一称为Sort-based Shuffle。</p>
<h4 id="2-1-Hash-Shuffle"><a href="#2-1-Hash-Shuffle" class="headerlink" title="2.1 Hash Shuffle"></a>2.1 Hash Shuffle</h4><p>在Spark 1.6.3之前，Hash Shuffle都是Spark Shuffle的解决方案之一。Shuffle的过程一般分为两个部分：Shuffle Write和Shuffle Fetch，前者是Map任务划分分区，输出中间结果，而后者则是Reduce任务获取到的这些中间结果。Hash Shuffle的过程如图下所示：<br><img src="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/FCF2BEB0-6A18-4EA0-9F22-46B0E3AA0952.png" alt="1b3f9d7e0de8684358ff919559d13a62"></p>
<blockquote>
<p>图中，Shuffle Write发生在一个节点上，执行shuffle任务的CPU核数为1，可以同时执行两个任务，每个任务输出的分区数与Reducer数相同，即为3。每个分区都有一个缓冲区（bucket）用来接收结果，每个缓冲区的大小由配置spark.shuffle.file.buffer.kb决定。这样每个缓冲区写满后，就会输出到一个文件段中。而Reducer就会去相应的节点拉取文件。</p>
</blockquote>
<p>这样设计起来其实是不复杂的。但问题也很明显，主要有两个：</p>
<ul>
<li><strong><em>生成的文件个数太大</em></strong>。理论上，每个Shuffle任务输出会产生R个文件（由Reduce个数决定），而Shuffle任务的个数往往是由Map任务个数M决定的，所以总共会生成M * R个中间结果文件，而在大型作业中，若是M和R都是很大的数字的话，就会出现文件句柄数突破操作系统的限制。</li>
<li><strong><em>缓冲区占用内存空间过大</em></strong>。 单节点在执行Shuffle任务时缓存区大小消耗(spark.shuffle.file.buffer.kb) × m × R , m为该节点运行的shuffle个数，如果一个核可以执行一个任务，那么m就与cpu核数相等。这对于有32，64核的服务器来说都是不小的内存开销。所有为了解决第一个问题，Spark引入了Flie Consolidation机制，指通过共同输出文件以降低文件数，如下图所示：</li>
</ul>
<p><img src="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/2F861E67-2AD8-422C-B3C0-5AF1885FBB60.png" alt="ac8c4e5d8e27ec9319930726a7f15068"></p>
<blockquote>
<p>每当Shuffle输出时，同一个CPU核心处理的Map任务的中间结果会输出到同分区的一个文件中，然后Reducer只需要一次性将整个文件拿到即可。这样，Shuffle产生的文件数为C（CPU核数）* R。Spark的FileConsolidation机制默认开启，可以通过spark.shuffle.consolidateFiles配置项进行配置。</p>
</blockquote>
<h4 id="2-2-Sort-based-Shuffle"><a href="#2-2-Sort-based-Shuffle" class="headerlink" title="2.2 Sort-based Shuffle"></a>2.2 Sort-based Shuffle</h4><p>即便是引入了FlieConsolidation后，还是无法根本解决中间文件数太大的问题，这时候Sort-based Shuffle才算是真正的引入进来。如图所示：<img src="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/3D2885AA-8879-41AF-8E7B-D5BA7CE09B9F.png" alt="7d856c507ee118374475fa61c694203b"></p>
<ul>
<li>每个Map任务会最后只输出两个文件（其中一个是索引文件），其中间过程采用MapReduce一样的归并排序，但是会用索引文件记录每个分区的偏移量，输出完成后，Reducer会根据索引文件得到属于自己的分区，这种情况下，shuffle产生的中间结果文件为2 * M（M为Map任务数）。</li>
<li>在基于排序的 Shuffle 中， Spark 还提供了一种折中方案——Bypass Sort-based Shuffle，当 Reduce 任务小于 spark.shuffle.sort.bypassMergeThreshold 配置（默认 200）时，Spark Shuffle 开始按照 Hash Shuffle 的方式处理数据，而不用进行归并排序，只是在 Shuffle Write 步骤的最后，将其合并为 1 个文件，并生成索引文件。这样实际上还是会生成大量的中间文件，只是最后合并为 1 个文件并省去排序所带来的开销，该方案的准确说法是 Hash Shuffle 的Shuffle Fetch 优化版。</li>
</ul>

      
      
        <div class="page-reward">
          <p><a href="javascript:void(0)" onclick="dashangToggle()" class="dashang">赏</a></p>
          <div class="hide_box"></div>
          <div class="shang_box">
            <a class="shang_close" href="javascript:void(0)" onclick="dashangToggle()">×</a>
            <div class="shang_tit">
              <p>纯属好玩</p>
            </div>
            <div class="shang_payimg">
              <img src="/img/alipayimg.jpg" alt="扫码支持" title="扫一扫" />
            </div>
              <div class="pay_explain">扫码打赏，你说多少就多少</div>
            <div class="shang_payselect">
              
                <div class="pay_item checked" data-id="alipay">
                  <span class="radiobox"></span>
                  <span class="pay_logo"><img src="/img/alipay.png" alt="支付宝" /></span>
                </div>
              
              
                <div class="pay_item" data-id="wechat">
                  <span class="radiobox"></span>
                  <span class="pay_logo"><img src="/img/weixin.png" alt="微信" /></span>
                </div>
              
            </div>
            <div class="shang_info">
              <p>打开<span id="shang_pay_txt">支付宝</span>扫一扫，即可进行扫码打赏哦</p>
            </div>
          </div>
        </div>
        <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/zepto/1.2.0/zepto.min.js"></script>
        <script type="text/javascript">
          $(".pay_item").click(function(){
            $(this).addClass('checked').siblings('.pay_item').removeClass('checked');
            var dataid=$(this).attr('data-id');
            $(".shang_payimg img").attr("src","/img/"+dataid+"img.jpg");
            $("#shang_pay_txt").text(dataid=="alipay"?"支付宝":"微信");
          });
          function dashangToggle(){

            $(".hide_box").fadeToggle();
            $(".shang_box").fadeToggle();
          }
        </script>
      
    </div>
    
  </div>
  
    
    <div class="copyright">
        <p><span>本文标题:</span><a href="">Spark-Shuffle原理</a></p>
        <p><span>文章作者:</span><a href="/" title="访问 月 迹 的个人博客">月 迹</a></p>
        <p><span>发布时间:</span>2020年10月14日 - 19时49分</p>
        <p><span>最后更新:</span>2020年10月14日 - 19时49分</p>
        <p>
            <span>原始链接:</span><a class="post-url" href="" title="Spark-Shuffle原理">https://xyueji.github.io/Spark-Shuffle原理.html</a>
            <span class="copy-path" data-clipboard-text="原文: https://xyueji.github.io/Spark-Shuffle原理.html　　作者: 月 迹" title="点击复制文章链接"><i class="fa fa-clipboard"></i></span>
            <script src="/js/clipboard.min.js"></script>
            <script> var clipboard = new Clipboard('.copy-path'); </script>
        </p>
        <p>
            <span>许可协议:</span><i class="fa fa-creative-commons"></i> <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/3.0/cn/" title="中国大陆 (CC BY-NC-SA 3.0 CN)" target = "_blank">"署名-非商用-相同方式共享 3.0"</a> 转载请保留原文链接及作者。
        </p>
    </div>



<nav id="article-nav">
  
    <a href="Spark解决数据倾斜问题.html" id="article-nav-newer" class="article-nav-link-wrap">
      <strong class="article-nav-caption"><</strong>
      <div class="article-nav-title">
        
          Spark解决数据倾斜问题
        
      </div>
    </a>
  
  
    <a href="MapReduce-Shuffle过程.html" id="article-nav-older" class="article-nav-link-wrap">
      <div class="article-nav-title">MapReduce-Shuffle过程</div>
      <strong class="article-nav-caption">></strong>
    </a>
  
</nav>

  
</article>

    <div id="toc" class="toc-article">
    <strong class="toc-title">文章目录</strong>
    <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#前言"><span class="toc-number">1.</span> <span class="toc-text">前言</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-RDD血缘与Spark容错"><span class="toc-number">1.1.</span> <span class="toc-text">1. RDD血缘与Spark容错</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1-1-窄依赖与宽依赖"><span class="toc-number">1.1.1.</span> <span class="toc-text">1.1 窄依赖与宽依赖</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#1-2-Spark的容错机制"><span class="toc-number">1.1.2.</span> <span class="toc-text">1.2 Spark的容错机制</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#2-Spark-Shuffle"><span class="toc-number">1.2.</span> <span class="toc-text">2. Spark Shuffle</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#2-1-Hash-Shuffle"><span class="toc-number">1.2.1.</span> <span class="toc-text">2.1 Hash Shuffle</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2-2-Sort-based-Shuffle"><span class="toc-number">1.2.2.</span> <span class="toc-text">2.2 Sort-based Shuffle</span></a></li></ol></li></ol></li></ol>
</div>
<input type="button" id="tocButton" value="隐藏目录"  title="点击按钮隐藏或者显示文章目录">

<script src="https://7.url.cn/edu/jslib/comb/require-2.1.6,jquery-1.9.1.min.js"></script>
<script>
    var valueHide = "隐藏目录";
    var valueShow = "显示目录";

    if ($(".left-col").is(":hidden")) {
        $("#tocButton").attr("value", valueShow);
    }
    $("#tocButton").click(function() {
        if ($("#toc").is(":hidden")) {
            $("#tocButton").attr("value", valueHide);
            $("#toc").slideDown(320);
        }
        else {
            $("#tocButton").attr("value", valueShow);
            $("#toc").slideUp(350);
        }
    })
    if ($(".toc").length < 1) {
        $("#toc, #tocButton").hide();
    }
</script>





<div class="bdsharebuttonbox">
	<a href="#" class="fx fa-weibo bds_tsina" data-cmd="tsina" title="分享到新浪微博"></a>
	<a href="#" class="fx fa-weixin bds_weixin" data-cmd="weixin" title="分享到微信"></a>
	<a href="#" class="fx fa-qq bds_sqq" data-cmd="sqq" title="分享到QQ好友"></a>
	<a href="#" class="fx fa-facebook-official bds_fbook" data-cmd="fbook" title="分享到Facebook"></a>
	<a href="#" class="fx fa-twitter bds_twi" data-cmd="twi" title="分享到Twitter"></a>
	<a href="#" class="fx fa-linkedin bds_linkedin" data-cmd="linkedin" title="分享到linkedin"></a>
	<a href="#" class="fx fa-files-o bds_copy" data-cmd="copy" title="分享到复制网址"></a>
</div>
<script>window._bd_share_config={"common":{"bdSnsKey":{},"bdText":"","bdMini":"2","bdMiniList":false,"bdPic":"","bdStyle":"2","bdSize":"24"},"share":{}};with(document)0[(getElementsByTagName('head')[0]||body).appendChild(createElement('script')).src='/static/api/js/share.js?v=89860593.js?cdnversion='+~(-new Date()/36e5)];</script>




    
        <link rel="stylesheet" href="https://cdn.bootcdn.net/ajax/libs/gitalk/1.6.2/gitalk.min.css">
<div id="gitalk-container" class="gitment-container"></div>
<script type="text/javascript">
    var gitalk = new Gitalk({
        clientID: '8864436c2b03c3753a80',
        clientSecret: '252abc963e694743b034294b00a36ec6c7ee2152',
        id: 'Spark-Shuffle原理.html',
        owner: 'xyueji',
        repo: 'xyueji.github.io',
        admin: 'xyueji'
    })
    gitalk.render('gitalk-container')
</script>

    



    <div class="scroll" id="post-nav-button">
        
            <a href="Spark解决数据倾斜问题.html" title="上一篇: Spark解决数据倾斜问题">
                <i class="fa fa-angle-left"></i>
            </a>
        
        <a title="文章列表"><i class="fa fa-bars"></i><i class="fa fa-times"></i></a>
        
            <a href="MapReduce-Shuffle过程.html" title="下一篇: MapReduce-Shuffle过程">
                <i class="fa fa-angle-right"></i>
            </a>
        
    </div>
    <ul class="post-list"><li class="post-list-item"><a class="post-list-link" href="Spark解决数据倾斜问题.html">Spark解决数据倾斜问题</a></li><li class="post-list-item"><a class="post-list-link" href="">Spark-Shuffle原理</a></li><li class="post-list-item"><a class="post-list-link" href="MapReduce-Shuffle过程.html">MapReduce-Shuffle过程</a></li><li class="post-list-item"><a class="post-list-link" href="Git开发规范.html">Git开发规范</a></li><li class="post-list-item"><a class="post-list-link" href="Flink学习--如何计算实时热门商品.html">Flink学习--如何计算实时热门商品</a></li><li class="post-list-item"><a class="post-list-link" href="Flink学习--基本概念与环境搭建.html">Flink学习--基本概念与环境搭建</a></li><li class="post-list-item"><a class="post-list-link" href="docker-compose搭建大数据环境.html">docker-compose搭建大数据环境</a></li><li class="post-list-item"><a class="post-list-link" href="Mac-docker动态给Container添加端口映射.html">Mac-docker动态给Container添加端口映射</a></li><li class="post-list-item"><a class="post-list-link" href="Vue+SpringBoot项目docker一键部署.html">Vue+SpringBoot项目docker一键部署</a></li><li class="post-list-item"><a class="post-list-link" href="vue-simple-uploader+springboot大文件上传.html">vue-simple-uploader+springboot大文件上传</a></li><li class="post-list-item"><a class="post-list-link" href="IDEA连接Impala.html">IDEA连接Impala</a></li><li class="post-list-item"><a class="post-list-link" href="HBase基本知识.html">HBase基本知识</a></li><li class="post-list-item"><a class="post-list-link" href="IDE破解.html">IDE破解</a></li><li class="post-list-item"><a class="post-list-link" href="salt-api配置.html">salt-api配置</a></li><li class="post-list-item"><a class="post-list-link" href="线上HBase表重建.html">线上HBase表重建</a></li><li class="post-list-item"><a class="post-list-link" href="Alluxio+Presto多版本测试.html">Alluxio+Presto多版本测试</a></li><li class="post-list-item"><a class="post-list-link" href="Linux-top命令详解.html">Linux-top命令详解</a></li><li class="post-list-item"><a class="post-list-link" href="Linux-free命令详解.html">Linux-free命令详解</a></li><li class="post-list-item"><a class="post-list-link" href="hbase的hbck工具.html">hbase的hbck工具</a></li><li class="post-list-item"><a class="post-list-link" href="JanusGraph.html">JanusGraph</a></li><li class="post-list-item"><a class="post-list-link" href="iptables禁用启用端口.html">iptables禁用启用端口</a></li><li class="post-list-item"><a class="post-list-link" href="Phoenix系统表.html">Phoenix系统表</a></li><li class="post-list-item"><a class="post-list-link" href="Phoenix安装问题.html">Phoenix安装问题</a></li><li class="post-list-item"><a class="post-list-link" href="Prometheus安装-HDFS指标收集.html">Prometheus安装+HDFS指标收集</a></li></ul>
    <script src="https://7.url.cn/edu/jslib/comb/require-2.1.6,jquery-1.9.1.min.js"></script>
    <script>
        $(".post-list").addClass("toc-article");
        $(".post-list-item a").attr("target","_blank");
        $("#post-nav-button > a:nth-child(2)").click(function() {
            $(".fa-bars, .fa-times").toggle();
            $(".post-list").toggle(300);
            if ($(".toc").length > 0) {
                $("#toc, #tocButton").toggle(200, function() {
                    if ($(".switch-area").is(":visible")) {
                        $("#tocButton").attr("value", valueHide);
                        }
                    })
            }
            else {
            }
        })
    </script>



    <script>
        
    </script>

</div>
      <footer id="footer">
    <div class="outer">
        <div id="footer-info">
            <div class="footer-left">
                &copy; 2020 月 迹
            </div>
            <div class="footer-right">
                <a href="http://hexo.io/" target="_blank" style="color: #000">Hexo</a>  Theme <a href="https://github.com/luuman/hexo-theme-spfk" target="_blank" style="color: #000">spfk</a> by luuman
            </div>
        </div>
        
            <div class="visit">
                
                    <span id="busuanzi_container_site_pv" style='display:none'>
                        <span id="site-visit" >本站到访数:
                            <span id="busuanzi_value_site_uv"></span>
                        </span>
                    </span>
                
                
                    <span>, </span>
                
                
                    <span id="busuanzi_container_page_pv" style='display:none'>
                        <span id="page-visit">本页阅读量:
                            <span id="busuanzi_value_page_pv"></span>
                        </span>
                    </span>
                
            </div>
        
    </div>
</footer>

    </div>
    <script src="https://7.url.cn/edu/jslib/comb/require-2.1.6,jquery-1.9.1.min.js"></script>
<script src="js/main.js"></script>

    <!--<script>-->
        <!--$(document).ready(function() {-->
            <!--var backgroundnum = 0;-->
            <!--var backgroundimg = "url(/background/bg-" + backgroundnum +".jpg)";-->
            <!--$("#mobile-nav").css({"background-image": backgroundimg,"background-size": "cover","background-position": "center"});-->
            <!--$(".left-col").css({"background-image": backgroundimg,"background-size": "cover","background-position": "center"});-->
        <!--})-->
    <!--</script>-->



<!-- Google Analytics -->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-168200248-1"></script>
<script>
    window.dataLayer = window.dataLayer || [];
    function gtag(){dataLayer.push(arguments);}
    gtag('js', new Date());

    gtag('config', 'UA-168200248-1');
</script>

<!-- End Google Analytics -->



<script>
	var _hmt = _hmt || [];
	(function() {
		var hm = document.createElement("script");
		hm.src = "https://hm.baidu.com/hm.js?1c0d5d76b7455fc584663ce315f75837";
		var s = document.getElementsByTagName("script")[0];
		s.parentNode.insertBefore(hm, s);
	})();
</script>



<div class="scroll" id="scroll">
    <a href="#"><i class="fa fa-arrow-up"></i></a>
    <a href="#comments"><i class="fa fa-comments-o"></i></a>
    <a href="#footer"><i class="fa fa-arrow-down"></i></a>
</div>
<script>
    $(document).ready(function() {
        if ($("#comments").length < 1) {
            $("#scroll > a:nth-child(2)").hide();
        };
    })
</script>

<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    
    <script language="javascript">
    $(function() {
        $("a[title]").each(function() {
            var a = $(this);
            var title = a.attr('title');
            if (title == undefined || title == "") return;
            a.data('title', title).removeAttr('title').hover(

            function() {
                var offset = a.offset();
                $("<div id=\"anchortitlecontainer\"></div>").appendTo($("body")).html(title).css({
                    top: offset.top - a.outerHeight() - 15,
                    left: offset.left + a.outerWidth()/2 + 1
                }).fadeIn(function() {
                    var pop = $(this);
                    setTimeout(function() {
                        pop.remove();
                    }, pop.text().length * 800);
                });
            }, function() {
                $("#anchortitlecontainer").remove();
            });
        });
    });
</script>
    
    
    
        <div class="search-result-list" ></div>
        <div class="search-cover"></div>
    <script type = "text/javascript" >
        $('#search').bind('keypress',function(event){
            if(event.keyCode == "13"){
                search()
            }
        });

    function search() {
        var searchTool = {
            data: [],
            arrList: [],
            oldList: [],
            init() {
                this.fetchJson()
            },
            fetchJson() {
                window.fetch('/content.json?t=' + (+new Date()), {
                    method: 'get',
                }).then((res) => {
                    return res.json()
                }).then((data) => {
                    this.data = data
                    let keywords = $('#search').val()
                    this.searchList(keywords)
                }).catch((err) => {
                    console.log(err)
                });
            },
            // 搜索字符串里面是否存在关键字
            isSreachIndexOF(oldstr, kw) {
                // console.log(oldstr, kw)
                var istrue = false
                // console.log('isSreachIndexOF', oldstr && toString.call(oldstr) === '[object Array]')
                if (oldstr && toString.call(oldstr) === '[object Array]') {
                    for (var i = 0; i < oldstr.length; i++) {
                        oldstr[i].toLowerCase() === kw.toLowerCase() ? istrue = true : null
                    }
                    return istrue
                }
                if (!oldstr || !kw) return false
                istrue = oldstr.toLowerCase().indexOf(kw.toLowerCase()) > -1
                return istrue
            },
            searchList(keywolds) {
                let divList = ''
                if (!keywolds) {
                    if (this.oldList.length) this.clear()
                } else {
                    this.data.forEach((item, index) => {
                        let tagsK = item.tags.map(item => item.name).join('-')
                        let categoriesK = item.categories.map(item => item.name).join('-')
                        if (this.isSreachIndexOF(item.title, keywolds) || this.isSreachIndexOF(tagsK, keywolds) || this.isSreachIndexOF(categoriesK, keywolds) || this.isSreachIndexOF(item.content, keywolds)) {
                            divList += `
                          <li>
                            <a href="/${item.path}" class="search-result-title">${item.title}</a>
                            <div class="main">
                              <div class="left">
                                <span>${item.author ? item.author : 'xyueji'}</span>
                                ${item.tags.map(tags => `<a href="/tags/${tags.slug}" class="search-result-tags">${tags.name}</a>`)}
                              </div>
                              <div class="right">${this.getFriendlyTime(item.date.replace('T', ' '), new Date())}</div>
                            </div>
                          </li>`
                        }
                    })
                    if (this.oldList.length) {
                        this.clear()
                    }
                    if (this.arrList.length) this.add()
                }
                if (!divList) {
                    divList = `<a class="search_close" href="javascript:void(0)" onclick="searchToggle()">×</a> <div style="text-align: center;font-size: 30px; margin-top: 50px;" ">抱歉，未搜索到相关结果</div>`
                } else {
                    divList = `<a class="search_close" href="javascript:void(0)" onclick="searchToggle()">×</a> <ul>` + divList + `</ul>`
                }
                $('.search-result-list').html(divList)
                $('.search-result-list').fadeToggle();
                $('.search-cover').fadeToggle();
            },
            clear() {
                this.oldList.forEach(item => {
                    $(".search-result-list li")[item].style.display = 'none'
                })
                this.oldList = []
            },
            add() {
                this.arrList.forEach(item => {
                    $(".search-result-list li")[item].style.display = 'block'
                })
                this.oldList = this.arrList
            },
            searchGo(keywolds) {
                this.arrList = []
                if (!keywolds) {
                    if (this.oldList.length) this.clear()
                } else {
                    this.data.forEach((item, index) => {
                        if (this.isSreachIndexOF(item.title, keywolds) || this.isSreachIndexOF(item.tags.map(item => item.name).join('-'), keywolds) || this.isSreachIndexOF(item.categories.map(item => item.name).join('-'), keywolds) || this.isSreachIndexOF(item.content, keywolds)) {
                            this.arrList.push(index)
                        }
                    })
                    if (this.oldList.length) this.clear()
                    if (this.arrList.length) this.add()
                }
            },
            /**
             * 获取指定时间的友好时间字符串。
             * @param str 指定的时间字符串，如yyyy-MM-dd HH:mm:ss
             * @param now 当前时间，允许时间戳，GMT时间，如果该参数为undefined，则使用浏览器时间。
             */
            getFriendlyTime(str, now) {
                var currentTime = new Date(now);
                var arr = str.split(/\s+/gi);
                var temp = 0, arr1, arr2, oldTime, delta;
                var getIntValue = function (ss, defaultValue) {
                    try {
                        return parseInt(ss, 10);
                    } catch (e) {
                        return defaultValue;
                    }
                };
                var getWidthString = function (num) {
                    return num < 10 ? ('0' + num) : num;
                };
                if (arr.length >= 2) {
                    arr1 = arr[0].split(/[\/\-]/gi);
                    arr2 = arr[1].split(':');
                    oldTime = new Date();
                    oldTime.setYear(getIntValue(arr1[0], currentTime.getFullYear()));
                    oldTime.setMonth(getIntValue(arr1[1], currentTime.getMonth() + 1) - 1);
                    oldTime.setDate(getIntValue(arr1[2], currentTime.getDate()));

                    oldTime.setHours(getIntValue(arr2[0], currentTime.getHours()));
                    oldTime.setMinutes(getIntValue(arr2[1], currentTime.getMinutes()));
                    oldTime.setSeconds(getIntValue(arr2[2], currentTime.getSeconds()));

                    delta = currentTime.getTime() - oldTime.getTime();

                    if (delta <= 60 * 1000) {
                        return '1分钟内';
                    } else if (delta < 60 * 60 * 1000) {
                        return Math.floor(delta / (60 * 1000)) + '分钟前';
                    } else if (delta < 24 * 60 * 60 * 1000) {
                        return Math.floor(delta / (60 * 60 * 1000)) + '小时前';
                    } else if (delta < 24 * 60 * 60 * 1000 * 30) {
                        return Math.floor(delta / (24 * 60 * 60 * 1000)) + '天前';
                    } else if (delta < 24 * 60 * 60 * 1000 * 30 * 12) {
                        return Math.floor(delta / (24 * 60 * 60 * 1000 * 30)) + '月前';
                    } else if (currentTime.getFullYear() != oldTime.getFullYear()) {
                        return [getWidthString(oldTime.getFullYear()), getWidthString(oldTime.getMonth() + 1), getWidthString(oldTime.getDate())].join('-')
                    } else {
                        return [getWidthString(oldTime.getMonth() + 1), getWidthString(oldTime.getDate())].join('-');
                    }
                }
                return '';
            }
        }
        searchTool.init()
    }
    function searchToggle() {
        $('.search-cover').fadeToggle();
        $('.search-result-list').fadeToggle();
    }
</script>


  </div>
</body>
</html>